import os
import shelve

#This is list of all skip terms
stopList = []

#This is db hash where we will store all word stats
db = shelve.open("wiki_wcstats.db")

def iterFile(fileName):
    """
    This is a file iterator
    """
    fileToRead = open(fileName)
    for eachLine in fileToRead:
        yield str(eachLine)

def readStopList():
    """
    This reads stop list terms for a file
    """
    global stopList
    fileToRead = open("stoplist.txt")
    for eachWord in fileToRead.readlines():
        eachWord = eachWord.strip()
        stopList.append(eachWord)
        
        
def genWcStats(someLine):
    """
    This generates word statistics
    """
    someLine = someLine.replace("--"," ")
    someLine = someLine.replace("-"," ")
    someLine = someLine.replace("," , " ")
    someLine = someLine.replace("." ," ")
    someLine = someLine.replace("("," ")
    someLine = someLine.replace(")"," ")

    wc_results = {}
    
    for eachWord in someLine.split():    
        eachWord = eachWord.replace(";","")
        eachWord = eachWord.replace("/","")
        eachWord = eachWord.replace("|","")

        if(eachWord not in stopList and eachWord != ""):
            eachWord = eachWord.lower()
            if(wc_results.has_key(eachWord) == False):
                wc_results[eachWord] = 0
            wc_results[eachWord] += 1
    return wc_results

def genWcStatsString(someDict):
    """
    This function is used to convert dictionary data to string
    we need this for shelve which only accepts strings 
    """
    result = ""
    for eachWord in someDict.keys():
        result += eachWord + ":"+ str(someDict[eachWord]) + "\t"
    return result 

def getTuples(someLine):
    """
    This parses text into structed list
    """
    if(len(someLine) > 7 and someLine.find("<title>")!= -1):
        titleString = someLine.split(">")[1]
        return ['title',titleString[titleString.find(":")+2:titleString.find("<")].replace(" ","_")]

    if(len(someLine) > 10 and someLine.find("<abstract>") != -1):
        abstract = someLine.replace("<abstract>","")
        abstract = abstract.replace("</abstract>","")
        return['abstract',genWcStatsString(genWcStats(abstract))]

    
if __name__ == "__main__":
    readStopList()
    fileIter = iterFile("abstract.xml")

    currentTitle = ""
    currentWcStats = ""
    currentCount = 0

    print "Stated Processing"
    
    for eachLine in fileIter:
        currentRecs = getTuples(eachLine.strip()) 

        if(currentCount > 10000 and currentCount % 10000 == 0):
            print currentCount
            
        if(currentRecs !=None and len(currentRecs) > 0):
            if(currentRecs[0] == 'title'):
                currentTitle = currentRecs[1]

            if(currentRecs[0] == 'abstract'):
                currentCount += 1
                
                currentWcStats = currentRecs[1]
                db[currentTitle] = currentWcStats
                currentTitle = ""

                currentWcStats = ""

    print "completed Processing" 
    db.close()
